<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.4.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/favicon.ico">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon.ico">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon.ico">
  <link rel="mask-icon" href="/images/favicon.ico" color="#222">

<link rel="stylesheet" href="/css/main.css">


<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"yoursite.com","root":"/","scheme":"Mist","version":"7.8.0","exturl":false,"sidebar":{"position":"left","display":"always","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":false,"show_result":false,"style":null},"back2top":{"enable":true,"sidebar":false,"scrollpercent":false},"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":false,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},"path":"search.xml"};
  </script>

  <meta name="description" content="如何把A网站的数据爬到B网站显示">
<meta property="og:type" content="article">
<meta property="og:title" content="如何把A网站的数据爬到B网站显示">
<meta property="og:url" content="http://yoursite.com/2021/05/16/tech-2021-05-16-%E5%A6%82%E4%BD%95%E6%8A%8AA%E7%BD%91%E7%AB%99%E6%95%B0%E6%8D%AE%E7%88%AC%E5%88%B0B%E7%BD%91%E7%AB%99%E6%98%BE%E7%A4%BA/index.html">
<meta property="og:site_name" content="Akiyama&#39;s Blog">
<meta property="og:description" content="如何把A网站的数据爬到B网站显示">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="http://yoursite.com/images/WebCrawler/csrfToken获取.png">
<meta property="article:published_time" content="2021-05-15T16:00:00.000Z">
<meta property="article:modified_time" content="2021-05-16T06:12:45.647Z">
<meta property="article:author" content="丘山月夜">
<meta property="article:tag" content="技术">
<meta property="article:tag" content="前端">
<meta property="article:tag" content="python">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="http://yoursite.com/images/WebCrawler/csrfToken获取.png">

<link rel="canonical" href="http://yoursite.com/2021/05/16/tech-2021-05-16-%E5%A6%82%E4%BD%95%E6%8A%8AA%E7%BD%91%E7%AB%99%E6%95%B0%E6%8D%AE%E7%88%AC%E5%88%B0B%E7%BD%91%E7%AB%99%E6%98%BE%E7%A4%BA/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh-CN'
  };
</script>

  <title>如何把A网站的数据爬到B网站显示 | Akiyama's Blog</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">Akiyama's Blog</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
      <p class="site-subtitle" itemprop="description">丘山月夜的博客</p>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="main-menu menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="home fa fa-home fa-fw"></i>首页</a>

  </li>
        <li class="menu-item menu-item-about">

    <a href="/about/" rel="section"><i class="user fa fa-user fa-fw"></i>关于</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="tags fa fa-tags fa-fw"></i>标签</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="th fa fa-th fa-fw"></i>分类</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="archive fa fa-archive fa-fw"></i>归档</a>

  </li>
  </ul>
</nav>




</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content post posts-expand">
            

    
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/05/16/tech-2021-05-16-%E5%A6%82%E4%BD%95%E6%8A%8AA%E7%BD%91%E7%AB%99%E6%95%B0%E6%8D%AE%E7%88%AC%E5%88%B0B%E7%BD%91%E7%AB%99%E6%98%BE%E7%A4%BA/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.png">
      <meta itemprop="name" content="丘山月夜">
      <meta itemprop="description" content="君は空を見てるか,<br>風の音を聞いてるか">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Akiyama's Blog">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          如何把A网站的数据爬到B网站显示
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              

              <time title="创建时间：2021-05-16 00:00:00 / 修改时间：14:12:45" itemprop="dateCreated datePublished" datetime="2021-05-16T00:00:00+08:00">2021-05-16</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/" itemprop="url" rel="index"><span itemprop="name"></span></a>
                </span>
            </span>

          
            <div class="post-description">如何把A网站的数据爬到B网站显示</div>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <h1 id="如何把A网站的数据爬到B网站显示"><a href="#如何把A网站的数据爬到B网站显示" class="headerlink" title="如何把A网站的数据爬到B网站显示"></a>如何把A网站的数据爬到B网站显示</h1><p>大体有两种思路</p>
<ol>
<li>使用python定期抓取数据，写入自己的数据库，再自己写个后端服务，前端请求自己的后端数据</li>
</ol>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">graph TB</span><br><span class="line">A网站--定期拉取数据--&gt;python脚本</span><br><span class="line">B网站--请求数据--&gt;后台服务</span><br><span class="line">subgraph 后端</span><br><span class="line">python脚本--保存数据--&gt;数据库</span><br><span class="line">后台服务--&gt;数据库</span><br><span class="line">end</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>不需要搭建后端和数据库，直接前端请求A网站后端接口数据</li>
</ol>
<p>这里采用第二种方式，坑有点多，下面一一介绍。</p>
<h1 id="过程"><a href="#过程" class="headerlink" title="过程"></a>过程</h1><h2 id="找到请求URL并模拟请求获取数据"><a href="#找到请求URL并模拟请求获取数据" class="headerlink" title="找到请求URL并模拟请求获取数据"></a>找到请求URL并模拟请求获取数据</h2><ol>
<li>控制台找到接口url和请求头等，postman模拟请求</li>
<li>由于后端有做校验，请求会失败，返回403或者500</li>
<li>从控制台中找到<code>Request Header</code>和<code>Cookie</code>等，复制到postman中模拟请求，成功说明爬数据可行。<ol>
<li>大部分header信息是没用的，可以一个一个删除，尝试请求，最后只留下关键的请求头和cookie。</li>
</ol>
</li>
<li>这里发现主要有两个信息，一个是csrfToken，一个是loginToken（实际名称不是这个）。<ol>
<li>csrfToken为自定义请求头，保存在headers中，错误服务端会返回500，并提示<code>invalid csrf token</code></li>
<li>loginToken保存在headers的cookie字段中，错误会返回200，提示登录失效，请重新登录</li>
<li>还有两个koa字段，用来配合csrfToken做校验，这三个值需要对应。由于保存在cookie中，获取方式和loginToken类似，因此不做详细说明</li>
</ol>
</li>
</ol>
<p><code>RequestHeader</code>关键信息如下：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> Cookie为浏览器自带的Header</span></span><br><span class="line">Cookie: loginToken=***; koa.xxx=***; koa.xxx.sig=***;</span><br><span class="line"><span class="meta">#</span><span class="bash"> csrf-token为服务端自定义Header</span></span><br><span class="line">csrf-token: *** </span><br></pre></td></tr></table></figure>

<p><strong>这里token可能会过期，我们不可能每次都去手动复制、粘贴请求头。因此需要动态抓取csrfToken和loginToken。</strong></p>
<p>控制台中可以查到在登录页面，后端返回的<code>Response Header</code>中的<code>Set-Cookie</code>字段带了loginToken，设置到了浏览器</p>
<p>csrfToken找了半天发现嵌在网页的body中。</p>
<p>我们的目标就是获取到该页面，通过正则从html中提取csrfToken。</p>
<img src="/images/WebCrawler/csrfToken获取.png" width='800'/>

<h2 id="Python模拟登录获取Cookie和Token等信息"><a href="#Python模拟登录获取Cookie和Token等信息" class="headerlink" title="Python模拟登录获取Cookie和Token等信息"></a>Python模拟登录获取Cookie和Token等信息</h2><blockquote>
<p><strong>可以先用postman模拟，验证可行性。</strong></p>
</blockquote>
<h3 id="获取loginToken"><a href="#获取loginToken" class="headerlink" title="获取loginToken"></a>获取loginToken</h3><ol>
<li>使用python请求登录页面url：需要设置相应的登录信息，如用户名和密码等。<ol>
<li>这里看到请求数据也是保存在cookie，而不是在<code>RequestBody</code>中，不过这一部分基本是不变的，因此可以直接复制粘贴cookie到<code>RequestHeader</code>中。</li>
</ol>
</li>
<li>登录成功后从<code>Response Header</code>中获取到相应的cookie，代码如下</li>
</ol>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> urllib.request</span><br><span class="line"><span class="keyword">import</span> http.cookiejar</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">login</span>():</span></span><br><span class="line">    <span class="keyword">global</span> loginToken</span><br><span class="line">    url = <span class="string">&quot;登录url&quot;</span></span><br><span class="line">    cookie = http.cookiejar.CookieJar()</span><br><span class="line">    handler = urllib.request.HTTPCookieProcessor(cookie)</span><br><span class="line">    opener = urllib.request.build_opener(handler)</span><br><span class="line">    opener.addheaders = [</span><br><span class="line">    		<span class="comment">#...</span></span><br><span class="line">      	<span class="comment"># 直接拷贝该url的cookie到Header中即可</span></span><br><span class="line">        (<span class="string">&#x27;Cookie&#x27;</span>, <span class="string">&#x27;...包含用户名之类的信息&#x27;</span>),</span><br><span class="line">        <span class="comment">#...</span></span><br><span class="line">    ]</span><br><span class="line">    request = urllib.request.Request(method=<span class="string">&quot;GET&quot;</span>, url=url)</span><br><span class="line">    response = opener.<span class="built_in">open</span>(request)</span><br><span class="line">    <span class="comment"># 获取ResponseHeader中的Cookie</span></span><br><span class="line">    <span class="keyword">for</span> item <span class="keyword">in</span> cookie:</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&#x27;cookies: %s = %s&#x27;</span> % (item.name, item.value))</span><br><span class="line">        <span class="keyword">if</span> item.name == <span class="string">&quot;loginToken&quot;</span>:</span><br><span class="line">            loginToken = item.value</span><br></pre></td></tr></table></figure>

<p>这里使用python的<code>requests</code>包请求会失败，原因应该是登录url有几次重定向。</p>
<blockquote>
<p> 使用<code>urllib.request + http.cookiejar</code>这两个包请求并获取cookie</p>
</blockquote>
<h3 id="获取csrfToken"><a href="#获取csrfToken" class="headerlink" title="获取csrfToken"></a>获取csrfToken</h3><p>请求html网页，使用正则提取csrfToken的值，代码如下</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">getKey</span>():</span></span><br><span class="line">    <span class="keyword">global</span> csrfToken</span><br><span class="line">    <span class="keyword">global</span> koa1</span><br><span class="line">    <span class="keyword">global</span> koa2</span><br><span class="line">    cookie = http.cookiejar.CookieJar()</span><br><span class="line">    handler = urllib.request.HTTPCookieProcessor(cookie)</span><br><span class="line">    opener = urllib.request.build_opener(handler)</span><br><span class="line">    response = opener.<span class="built_in">open</span>(<span class="string">&#x27;请求url&#x27;</span>)</span><br><span class="line">    <span class="comment"># read读取结果</span></span><br><span class="line">    code_of_html = response.read().decode(<span class="string">&#x27;utf-8&#x27;</span>)</span><br><span class="line">		<span class="comment"># 正则提取CSRFToken字段的值</span></span><br><span class="line">    csrfToken = re.search(<span class="string">&#x27;&#x27;&#x27;(?&lt;=CSRFToken&amp;quot;:&amp;quot;).*?(?=&amp;quot;)&#x27;&#x27;&#x27;</span>, code_of_html).group()</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;csrfToken: &quot;</span> + csrfToken)</span><br><span class="line">    <span class="keyword">for</span> item <span class="keyword">in</span> cookie:</span><br><span class="line">      	<span class="comment"># 这里其实除了csrfToken之外还有两个保存在cookie字段，是用来配合csrfToken做校验的。由于获取方式和loginToken类似，因此不做详细说明</span></span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&#x27;cookies: %s = %s&#x27;</span> % (item.name, item.value))</span><br><span class="line">        <span class="keyword">if</span> item.name == <span class="string">&quot;koa.***&quot;</span>:</span><br><span class="line">            koa1 = item.value</span><br><span class="line">        <span class="keyword">if</span> item.name == <span class="string">&quot;koa.***.sig&quot;</span>:</span><br><span class="line">            koa2 = item.value</span><br></pre></td></tr></table></figure>

<h2 id="python请求接口数据"><a href="#python请求接口数据" class="headerlink" title="python请求接口数据"></a>python请求接口数据</h2><p>获取到token信息之后，添加到相应的请求头中即可，模拟请求成功，结果同postman。</p>
<p>走到这一步方案一和方案二还是一样的。用python请求数据先验证可行性，后面实在走不通也可以切换到方案一，将python爬到的数据保存到数据库。</p>
<p>事实上，由于前端浏览器环境和直接用python有很大的不同，后面会有很多坑。</p>
<h2 id="前端请求接口数据"><a href="#前端请求接口数据" class="headerlink" title="前端请求接口数据"></a>前端请求接口数据</h2><p>这里先不管请求头怎么设置，直接请求数据，可以使用Chrome ModHeader插件手动填写请求头。</p>
<h3 id="浏览器跨域请求问题"><a href="#浏览器跨域请求问题" class="headerlink" title="浏览器跨域请求问题"></a>浏览器跨域请求问题</h3><ol>
<li>方法1：服务器设置允许跨域。由于是爬别人的数据，没有后端，因此走不通。</li>
<li>方法2：使用代理，将请求转发到代理服务，由代理服务拿到结果后返回给浏览器。</li>
</ol>
<p>以vue+vite配置为例：</p>
<figure class="highlight js"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">//vite.config.js配置代理</span></span><br><span class="line">server: &#123;</span><br><span class="line">	<span class="comment">//配置代理解决跨域问题</span></span><br><span class="line">	proxy: &#123;</span><br><span class="line">		<span class="string">&quot;/api&quot;</span>: &#123;</span><br><span class="line">			target: <span class="string">&quot;https://hostname.com/&quot;</span>, <span class="comment">// 你请求的第三方接口</span></span><br><span class="line">			changeOrigin: <span class="literal">true</span>, <span class="comment">// 在本地会创建一个虚拟服务端，然后发送请求的数据，并同时接收请求的数据，这样服务端和服务端进行数据的交互就不会有跨域问题</span></span><br><span class="line">			rewrite: <span class="function">(<span class="params">path</span>) =&gt;</span> path.replace(<span class="regexp">/^\/api/</span>, <span class="string">&quot;&quot;</span>),</span><br><span class="line">		&#125;,</span><br><span class="line">	&#125;,</span><br><span class="line">&#125;,</span><br></pre></td></tr></table></figure>

<h4 id="vite和nginx部署"><a href="#vite和nginx部署" class="headerlink" title="vite和nginx部署"></a>vite和nginx部署</h4><p>vite和nginx都可以作为Web服务器运行前端页面，也支持配置代理服务。vite内部还是通过nodejs启动服务器</p>
<p>在我看来，他们的区别只在于<strong>nginx是专业的服务器，性能更好。而vite是一个开发工具</strong>。（PS：如负载均衡、并发强、支持反向代理等，这一块没接触过不懂）</p>
<p>一般情况下vite只在开发时使用，正式环境需要编译打包代码，部署到nginx之类的专业服务器，这个时候vite的配置就不生效了，因此再使用nginx做反向代理。</p>
<p>考虑到内部使用的小型网站，直接使用vite部署即可。</p>
<figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">//package.json</span></span><br><span class="line">&#123;</span><br><span class="line">  <span class="attr">&quot;scripts&quot;</span>: &#123;</span><br><span class="line">    <span class="attr">&quot;build&quot;</span>: <span class="string">&quot;vite build&quot;</span>,</span><br><span class="line">    <span class="attr">&quot;serve&quot;</span>: <span class="string">&quot;vite preview&quot;</span></span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">//npm run build &amp; npm run serve运行</span></span><br></pre></td></tr></table></figure>

<h3 id="Axios设置请求头问题"><a href="#Axios设置请求头问题" class="headerlink" title="Axios设置请求头问题"></a>Axios设置请求头问题</h3><h4 id="Refused-to-set-unsafe-header-“Cookie”"><a href="#Refused-to-set-unsafe-header-“Cookie”" class="headerlink" title="Refused to set unsafe header “Cookie”"></a>Refused to set unsafe header “Cookie”</h4><p>axios请求配置headers报错： Refused to set unsafe header “Cookie”。</p>
<blockquote>
<p>w3c规定，添加以下不安全的请求头，浏览器会终止请求。</p>
</blockquote>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line">Accept-Charset</span><br><span class="line">Accept-Encoding</span><br><span class="line">Connection</span><br><span class="line">Content-Length</span><br><span class="line">Cookie</span><br><span class="line">Cookie2</span><br><span class="line">Content-Transfer-Encoding</span><br><span class="line">Date</span><br><span class="line">Expect</span><br><span class="line">Host</span><br><span class="line">Keep-Alive</span><br><span class="line">Referer</span><br><span class="line">TE</span><br><span class="line">Trailer</span><br><span class="line">Transfer-Encoding</span><br><span class="line">Upgrade</span><br><span class="line">User-Agent</span><br><span class="line">Via</span><br></pre></td></tr></table></figure>

<p>服务端自定义的请求头csrfToken不会有问题。因此关键信息只有cookie。</p>
<p>解决方案：</p>
<blockquote>
<p>js使用document.cookie给浏览器设置cookie，请求的时候浏览器会自动带上</p>
</blockquote>
<p><strong>如果还有其他不安全的Header需要设置，暂时不知道怎么解决，这取决于服务端</strong></p>
<p>实在走不通就只能换回方案一，或者通过Mod Header手动填写Headers。由于token会过期，因此隔一段时间就要改很麻烦</p>
<h4 id="带Http-Only的cookie，无法通过document-cookie获取和设置"><a href="#带Http-Only的cookie，无法通过document-cookie获取和设置" class="headerlink" title="带Http Only的cookie，无法通过document.cookie获取和设置"></a>带Http Only的cookie，无法通过document.cookie获取和设置</h4><p>参考文章：<a target="_blank" rel="noopener" href="https://zhuanlan.zhihu.com/p/36197012">浅谈Js 操作Cookie，以及HttpOnly 的限制</a></p>
<p>简单来说，后端通过<code>ResponseHeader</code>的<code>Set-Cookie</code>字段设置浏览器Cookie，如果带了httponly标志，表示该cookie只能通过服务端修改，客户端浏览器无法通过JS的document.cookie获取和设置该cookie。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">Set-Cookie: loginToken=***; path=/; httponly</span><br></pre></td></tr></table></figure>

<blockquote>
<p>HttpOnly是包含在Set-Cookie HTTP响应头文件中的附加标志。生成cookie时使用HttpOnly标志有助于降低客户端脚本访问受保护cookie的风险（如果浏览器支持）</p>
</blockquote>
<p>解决方案：</p>
<blockquote>
<p>浏览器设置退出的时候清除该网站cookie。</p>
<p>原因：网页首次加载的时候可以设置该Cookie，因为该httponly的标志是服务端Response的时候设置的。只要清除该Cookie，每次打开网页通过document.cookie设置即可。</p>
</blockquote>
<h2 id="前端模拟登录动态获取请求头和Token"><a href="#前端模拟登录动态获取请求头和Token" class="headerlink" title="前端模拟登录动态获取请求头和Token"></a>前端模拟登录动态获取请求头和Token</h2><p>这里存在很多问题：</p>
<ol>
<li>跨域问题</li>
<li>登录url重定向：前端请求不知道支不支持处理重定向后的结果</li>
<li>用户信息加密：如果不知道加密算法，即使输入了账号密码也不行</li>
<li>4A登录：大部分公司有单点登录的门户，单点登录返回的token，不是实际业务网站的token，因此即使登录了也没法通过这个token请求接口数据。</li>
</ol>
<p>这些问题前端比较难解决，除非知道正式的登录接口如何请求。因此最终选了其他方案。</p>
<blockquote>
<p>前面通过python模拟登录，可以拿到token等信息，只要自己写个接口，返回token信息的json即可。</p>
<p>取巧的方式：不需要写代码和接口，直接把token写到一个文件中，使用python内置的http服务器开启服务端口，访问该文件即可。</p>
<ul>
<li>Python2：<code>python -m SimpleHTTPServer 8081 </code>，指定端口8081，默认端口8080</li>
<li>Python3：<code>python -m http.server 8081</code></li>
</ul>
</blockquote>
<h2 id="结束"><a href="#结束" class="headerlink" title="结束"></a>结束</h2><p>到这里前端网页已经能够拿到其他网站的数据了，具体拿到这个数据怎么处理，怎么组装就要写逻辑了。</p>
<p>别人网站的接口数据格式一般情况下就只适用于自身的UI和业务，放到我们自己的网站UI肯定需要组装和适配。</p>
<p>总结：一步一个坑</p>

    </div>

    
    
    
        

  <div class="followme">
    <p>欢迎关注我的其它发布渠道</p>

    <div class="social-list">

        <div class="social-item">
          <a target="_blank" class="social-link" href="/images/wechat_channel.jpg">
            <span class="icon">
              <i class="fab fa-weixin"></i>
            </span>

            <span class="label">WeChat</span>
          </a>
        </div>
    </div>
  </div>


      <footer class="post-footer">
          
          <div class="post-tags">
              <a href="/" rel="tag"><i class="fa fa-tag"></i> </a>
              <a href="/" rel="tag"><i class="fa fa-tag"></i> </a>
              <a href="/" rel="tag"><i class="fa fa-tag"></i> </a>
          </div>

        


        
    <div class="post-nav">
      <div class="post-nav-item">
    <a href="/2021/05/16/webdev-2021-05-16-%E5%89%8D%E7%AB%AF%E7%BD%91%E7%AB%99%E9%83%A8%E7%BD%B2/" rel="prev" title="前端网站部署">
      <i class="fa fa-chevron-left"></i> 前端网站部署
    </a></div>
      <div class="post-nav-item"></div>
    </div>
      </footer>
    
  </article>
  
  
  



          </div>
          

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="header-overlay"></div>
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#%E5%A6%82%E4%BD%95%E6%8A%8AA%E7%BD%91%E7%AB%99%E7%9A%84%E6%95%B0%E6%8D%AE%E7%88%AC%E5%88%B0B%E7%BD%91%E7%AB%99%E6%98%BE%E7%A4%BA"><span class="nav-number">1.</span> <span class="nav-text">如何把A网站的数据爬到B网站显示</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#%E8%BF%87%E7%A8%8B"><span class="nav-number">2.</span> <span class="nav-text">过程</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#%E6%89%BE%E5%88%B0%E8%AF%B7%E6%B1%82URL%E5%B9%B6%E6%A8%A1%E6%8B%9F%E8%AF%B7%E6%B1%82%E8%8E%B7%E5%8F%96%E6%95%B0%E6%8D%AE"><span class="nav-number">2.1.</span> <span class="nav-text">找到请求URL并模拟请求获取数据</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Python%E6%A8%A1%E6%8B%9F%E7%99%BB%E5%BD%95%E8%8E%B7%E5%8F%96Cookie%E5%92%8CToken%E7%AD%89%E4%BF%A1%E6%81%AF"><span class="nav-number">2.2.</span> <span class="nav-text">Python模拟登录获取Cookie和Token等信息</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#%E8%8E%B7%E5%8F%96loginToken"><span class="nav-number">2.2.1.</span> <span class="nav-text">获取loginToken</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E8%8E%B7%E5%8F%96csrfToken"><span class="nav-number">2.2.2.</span> <span class="nav-text">获取csrfToken</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#python%E8%AF%B7%E6%B1%82%E6%8E%A5%E5%8F%A3%E6%95%B0%E6%8D%AE"><span class="nav-number">2.3.</span> <span class="nav-text">python请求接口数据</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E5%89%8D%E7%AB%AF%E8%AF%B7%E6%B1%82%E6%8E%A5%E5%8F%A3%E6%95%B0%E6%8D%AE"><span class="nav-number">2.4.</span> <span class="nav-text">前端请求接口数据</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#%E6%B5%8F%E8%A7%88%E5%99%A8%E8%B7%A8%E5%9F%9F%E8%AF%B7%E6%B1%82%E9%97%AE%E9%A2%98"><span class="nav-number">2.4.1.</span> <span class="nav-text">浏览器跨域请求问题</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#vite%E5%92%8Cnginx%E9%83%A8%E7%BD%B2"><span class="nav-number">2.4.1.1.</span> <span class="nav-text">vite和nginx部署</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#Axios%E8%AE%BE%E7%BD%AE%E8%AF%B7%E6%B1%82%E5%A4%B4%E9%97%AE%E9%A2%98"><span class="nav-number">2.4.2.</span> <span class="nav-text">Axios设置请求头问题</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Refused-to-set-unsafe-header-%E2%80%9CCookie%E2%80%9D"><span class="nav-number">2.4.2.1.</span> <span class="nav-text">Refused to set unsafe header “Cookie”</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#%E5%B8%A6Http-Only%E7%9A%84cookie%EF%BC%8C%E6%97%A0%E6%B3%95%E9%80%9A%E8%BF%87document-cookie%E8%8E%B7%E5%8F%96%E5%92%8C%E8%AE%BE%E7%BD%AE"><span class="nav-number">2.4.2.2.</span> <span class="nav-text">带Http Only的cookie，无法通过document.cookie获取和设置</span></a></li></ol></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E5%89%8D%E7%AB%AF%E6%A8%A1%E6%8B%9F%E7%99%BB%E5%BD%95%E5%8A%A8%E6%80%81%E8%8E%B7%E5%8F%96%E8%AF%B7%E6%B1%82%E5%A4%B4%E5%92%8CToken"><span class="nav-number">2.5.</span> <span class="nav-text">前端模拟登录动态获取请求头和Token</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E7%BB%93%E6%9D%9F"><span class="nav-number">2.6.</span> <span class="nav-text">结束</span></a></li></ol></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image" alt="丘山月夜"
      src="/images/avatar.png">
  <p class="site-author-name" itemprop="name">丘山月夜</p>
  <div class="site-description" itemprop="description">君は空を見てるか,<br>風の音を聞いてるか</div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">60</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">14</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">21</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>



      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        

<div class="copyright">
  
  &copy; 
  <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">丘山月夜</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://mist.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Mist</a> 强力驱动
  </div>

        








      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>

<script src="/js/utils.js"></script>

<script src="/js/motion.js"></script>


<script src="/js/schemes/muse.js"></script>


<script src="/js/next-boot.js"></script>




  















  

  

</body>
</html>
